import math
import os

NUMBER_OF_MACHINES = 13.0
URL_FILE_NAME = "z02-url"
print "simple tool for spliting the URL file"

#cmd_rm = "rm z01-url-*"
#os.system( cmd_rm )

inputFileHandler1 = open("/data/weijiang/clusterPageDownloading/z02-url-related/z02-url","r")

URLs = inputFileHandler1.readlines()
numberOfURLs = len(URLs)

numberOfURLsPerMachine = math.ceil( numberOfURLs/NUMBER_OF_MACHINES )
#numberOfURLsPerMachine = numberOfURLs/NUMBER_OF_MACHINES 

print "numberOfURLs:",numberOfURLs
print "numberOfURLsPerMachine:",numberOfURLsPerMachine
print "numberOfURLsProcessed(Estimated):",numberOfURLsPerMachine*NUMBER_OF_MACHINES

offset = 0
upperBound = numberOfURLsPerMachine
for NOTRecordedCounter in range( 1,int(NUMBER_OF_MACHINES+1) ):
    outputQueryFeatureFileHandler = open("/data/weijiang/clusterPageDownloading/z02-url-related/z02-url-part" + "%02d" % NOTRecordedCounter + "OutOf" + str(int(NUMBER_OF_MACHINES)),"w")
    
    print 
    print "part:",str(NOTRecordedCounter),"OutOf",NUMBER_OF_MACHINES
    print "offset:",str(offset)
    print "upperBound:",str(upperBound)
    
    for eachLine in URLs[int(offset):int(upperBound)]:
        outputQueryFeatureFileHandler.write(eachLine)
    outputQueryFeatureFileHandler.close()
    offset += numberOfURLsPerMachine
    upperBound += numberOfURLsPerMachine
    
inputFileHandler1.close()